package com.jiuzhou.crawlers.taobao.processor.taobao;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.fs.Path;

import com.jiuzhou.crawlers.taobao.processor.taobao.constant.CommonConstant;
import com.jiuzhou.crawlers.taobao.util.DataGetUtil;
import com.jiuzhou.crawlers.taobao.util.HdfsUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Json;

public class CommentsProcessor  implements PageProcessor{
	 private Site site =  Site.me().setRetryTimes(3).setSleepTime(500).addHeader("referer", "https://item.taobao.com/item.htm");  //站点设置
	
		public static Map<String,List<Integer>> brandLineCount =  new HashMap<String,List<Integer>>(); // Map<文件名, List<每行商品数目>> 
		
		public  List<List<Path>> allPaths ;
	 
		public static boolean isCurrentType = false;
	@Override
	public Site getSite() {
		return site;
	}

	
	/**方法名称：process()
	 * 方法说明：评论爬取processor
	 */
	@Override
	public void process(Page page) {
		//链接是否匹配评论
		   if(page.getUrl().regex("https://rate.taobao.com/feedRateList.htm").match()){
			   //截取id
           	int idBegain = page.getUrl().toString().indexOf("=")+1;
           	int idEnd = page.getUrl().toString().indexOf("&");
            String itemId  = page.getUrl().toString().substring(idBegain,idEnd);
           	   //截取json
          	int jsonBegain = page.getRawText().toString().indexOf("(")+1;
           	int jsonEnd = page.getRawText().toString().lastIndexOf(")");
         	String jsonString =  page.getRawText().toString().substring(jsonBegain,jsonEnd);
           	Json json = new Json(jsonString); 
           	   //截取当前页
         	int pageBegin = page.getUrl().toString().indexOf("currentPageNum")+"currentPageNum=".length();
           	int pageEnd = page.getUrl().toString().lastIndexOf("&pageSize");
            
            List<String > comNum =  json.jsonPath("$.comments[*]").all();
          
            page.putField("itemId", itemId);
            page.putField("conments", jsonString);
            page.putField("currentPageNum", page.getUrl().toString().substring(pageBegin,pageEnd));
            page.putField("total", comNum.size());
            
            //先提取商品总共有多少页面，再添加相应链接
            int  total =Integer.parseInt(json.jsonPath("$.total").toString()) ;
            if(total>20){
                int restPage = total%20 == 0 ? 0 : 1;
                int pages = total/20 + restPage;
  
                    for(int i = 2 ; i <= pages ; i++){
                    	if(i <= 250){
                    	    page.addTargetRequest("https://rate.taobao.com/feedRateList.htm?auctionNumId="+itemId+"&orderType=feedbackdate&currentPageNum="+i+"&pageSize=20");
                    	}
                    }


            }
  
		   }else{
			   page.setSkip(true);
		   }
	}

}
